# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
# from itemadapter import ItemAdapter
import os
import requests

TARGET = os.path.join(os.path.expanduser('~'), 'Downloads/chanspider/24-04-22')


class Scrapper4ChansPipeline:
    def process_item(self, item, spider):
        folder = os.path.join(TARGET, item["folder"])

        try:
            os.makedirs(folder)
        except Exception as e:
            print(
                f'Failed to create folder')
            print(e)

        try:
            for target in item["hrefs"]:
                r = requests.get(target["link"])
                with open(os.path.join(folder, target["filename"]), 'wb') as f:
                    f.write(r.content)
        except Exception as e:
            print(e)

        return item


class TextChanPipeline:
    def process_item(self, item, spider):
        folder = os.path.join(os.path.expanduser(
            '~'), 'Downloads/textchan/23-06-21')

        try:
            os.makedirs(folder)
        except Exception as e:
            print(
                f'Failed to create folder')
            print(e)

        try:
            with open(os.path.join(folder, item["filename"]), 'w') as f:
                f.write(item["posts"])
        except Exception as e:
            print(e)
            print('HITIHTIHTI')

        return item
